'use strict'

// var initDb=require('./utils/initDb');

// initDb();

//var Crawler = require('crawler');
var article = require('./models/articles');

var Crawler = require("crawler");
 
var crawler = new Crawler({
    maxConnections : 10,
    // This will be called for each crawled page
    callback : function (error, res, done) {
        if(error){
            console.log(error);
        }else{
            var $ = res.$; //cheerio
            console.log($('title').text());
            var data = $('.dw_table .el').toArray(); //51
            data.forEach(element => {
                var jobName = $(element).find('.t1 a').attr('title');
                if (jobName == '数据分析师') {
                    var company = $(element).find('.t2 a').text().trim().replace(/\s/g, '');
                    var Address = $(element).find('.t3').text().trim().replace(/\s/g, '');
                    var  Salary = $(element).find('.t4').text().trim().replace(/\s/g, '');
                    var Releasetime = $(element).find('.t5').text().trim().replace(/\s/g, '');
                    console.log(jobName, company,Address,Salary,Releasetime);
                    article.sync({
                        force: true
                    }).then(() => {
                        article.create({
                            jobName: jobName,
                            company: company,
                            Address: Address,
                            Salary: Salary,
                            Releasetime: Releasetime
 
                        })
                    })
                }
            });
        }
        done(); //爬取完成
    }
});



for(let index = 1 ;index <=10 ;index +=1){
    crawler.queue(`https://search.51job.com/list/000000,000000,0000,00,9,99,%25E6%2595%25B0%25E6%258D%25AE%25E5%2588%2586%25E6%259E%2590%25E5%25B8%2588,2,${index}.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=`);
}